#!/bin/bash

# check and kill long running processes on the mentat systems
# this script should run on all systems as root
#
#    (c)2011 Donders Centre/Simon Oosthoek
#
#    This file is part of PIM.
#
#    PIM is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 2 of the License, or
#    (at your option) any later version.
#
#    PIM is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with PIM.  If not, see <http://www.gnu.org/licenses/>.
#

TESTMODE=false
if [ $# -gt 0 ]
then
	TESTMODE=true
fi

# check if running as root, otherwise warn!
if [ $USER != "root" ]
then
	echo "$PROG: warning - not running as root, only $USER's processes will be killed" >/dev/null
fi

if [ -r /opt/cluster/pim_lib ]
then
	. /opt/cluster/pim_lib
else
	echo "$PROG: cannot source /opt/cluster/pim_lib" >&2
	exit 1
fi

if [ -f $pim_conffile ]
then
	. $pim_conffile
else
	echo "$PROG: cannot source $pim_conffile" >&2
	exit 1
fi

mailtempl_userkill=$pim_configdir/mail_userkill.tpl
mailtempl_kill=$pim_configdir/mail_kill.tpl

function fill_template {
	templ="$1"
	subject="$2"
	to_email="$3"
	days=$(echo $pid_elapsedtime|sed 's/-.*$//')
	url_hostinfo="${wui_urlbase}/pidinfo.php?host=$pid_machine"
	cat $mailtempl_longrun | 
	sed 	-e "/__subject__/s//$subject/g" \
		-e "/__to_email__/s//$to_email/g" \
		-e "/__user__/s//$pid_ownername/g" \
		-e "/__procname__/s//$procname/g" \
		-e "/__pid_machine__/s//$pid_machine/g" \
		-e "/__pid_pid__/s//$pid_pid/g" \
		-e "/__pid_owner__/s//$pid_owner/" \
		-e "/__time_notified__/s//$time_notified/g"
}


function sendmail_userkill {
	tpl="$mailtempl_userkill"
	subject="[PIM] user requested kill on $pid_machine" 
	to_email="$pid_owner"
	if fill_template "$tpl" "$subject" "$to_email" >/dev/null
	then
		TESTINGVAL=" TESTING"
		if [ $TESTMODE = false ]
		then
			TESTINGVAL=""
			fill_template "$tpl" "$subject" "$to_email" | 
			$SENDMAIL -t 
		fi
		#subject="[PIM] admincopy user requested kill on ${pid_machine}${TESTINGVAL}" 
		#to_email="$ADMINMAIL"
		#fill_template "$tpl" "$subject" "$to_email" | 
		#	$SENDMAIL -t 
	else
		echo "$PROG: error filling template for e-mail" >&2
	fi
}

function sendmail_killed {
	tpl="$mailtempl_kill"
	subject="[PIM] process killed on $pid_machine"
	to_email="$pid_owner"
	if fill_template "$tpl" "$subject" "$to_email" >/dev/null
	then
		if [ $TESTMODE = false ]
		then
			fill_template "$tpl" "$subject" "$to_email" | 
			$SENDMAIL -t 
		fi
		#subject="[PIM] admincopy process killed on $pid_machine"
		#to_email="$ADMINMAIL"
		#fill_template "$tpl" "$subject" "$to_email" | 
		#	$SENDMAIL -t 
	else
		echo "$PROG: error filling template for e-mail" >&2
	fi
}

function sendmail_debug {
	subject="$1"
	to_email="$ADMINMAIL"
	cat <<EOF | $SENDMAIL -t 
Subject: $subject
To: $to_email
From: root@$MACHINE

       state=$state
         pid=$pid_pid
     machine=$pid_machine
pid_notified="$pid_notified" $(epoch2date $pid_notified)
 pid_renewed="$pid_renewed" $(epoch2date $pid_renewed)
  user_renew="$user_renew" $(epoch2date $user_renew)
 pid_randval="$pid_randval"
    user_key="$user_key"
   user_kill="$user_kill" $(epoch2date $user_kill)
EOF
}

# check all pidfiles for this machine
for pidfile in $pim_datadir/$MACHINE*
do
	if [ ! -f "$pidfile" ]
	then
		# probably no files exist for this machine
		echo "no pidfiles" >/dev/null
		break
	fi

	#make sure we reset variables which we test for and which are not
	#always overwritten 
	pid_ownername="" 
	pid_notified="0"
	pid_randval=""
	user_key=""
	pid_renewed="0"
	pid_excluded="0"
	user_renew="0"
	user_kill="0"

	# source the pid_file
	. "$pidfile"

	# source user interaction file, if present
	actionfile=$pim_useract/$(basename $pidfile)
	if [ -f $actionfile ]
	then
		. $actionfile
	fi

	# check if the process is still active
        if ps -p $pid_pid -o pid= -o cmd= &>/dev/null
        then
                echo "process is still active" >/dev/null
        else
		echo "$pid_pid, process not there?" >/dev/null
		continue
	fi

	# skip if user never notified (AFAWK)
	if [ -z "$pid_notified" ]
	then
		echo user not notified yet >/dev/null
		continue
	fi

	# test whether user must be excluded from the kill regime
	if exclude_user $pid_owner
	then
		continue
	elif [ $pid_excluded -gt $pid_notified ]
	then
		# user was in excluded state, so needs a notificatio before kill
		continue
	fi
		

	if [ -f "$pid_configused" ]
	then
		# source the config file used	
		. "$pid_configused"
	else
		#echo "$PROG: error: $pid_configused not found for $pidfile" >&2
		continue
	fi

	#skip if a user filter exists and applies to this user
	if [ -n "$only_for_groups" ]
	then
		if id -Gn $pid_owner|grep -Ew "$only_for_groups" >/dev/null
		then
			echo "user is member of a monitored group" >/dev/null
		else
			#echo "user $pid_owner group-ignored ($pid_machine $pid_pid $pid_elapsedtime)" >>$mailreport
			continue
		fi
	fi

	#skip if a user filter exists and applies to this user
	if [ -n "$only_for_users" ]
	then
		#check if owner in list of monitorable users
		if echo $pid_owner|grep -Ew "$only_for_users" >/dev/null
		then
			echo "user's processes will be managed" >/dev/null
		else
			#echo "user $pid_owner user-ignored ($pid_machine $pid_pid $pid_elapsedtime)" >>$mailreport
			continue
		fi
	fi

	#determine starttime for this process
	if [ -n "$pid_starttime" ]
	then
		start=$pid_starttime
	else
		etime_secs=$(etime2sec "$pid_elapsedtime")
		start=$(($pid_timestamp-$etime_secs))
	fi

	if [ "$user_renew" -gt 0 ]
	then	
		if [ $pid_renewed -lt $user_renew ]
		then 
			if [ $pid_randval = $user_key ]
			then
				# renew verified, set pid_renewed and renew process
				pid_renewed=$user_renew
				if [ $TESTMODE = false ]
				then
					renew_pid  "$pidfile"  $user_renew
				fi
			else 
				sendmail_debug "[PIM] killer: renew failed, key mismatch" 
			fi
		fi
	fi
	
	#process state machine
	# states: start, notified, renewed, kill, userkill (end)
	state="unknown"
	if [ "$pid_notified" -eq 0 ]
	then
		#state="start"
		echo "$pid_pid, state=initial, not notified yet" >/dev/null
		continue
	elif [ $pid_notified -gt $pid_renewed ]
	then
		state="notified"
	elif [ $pid_renewed -gt $pid_notified ]
	then
		state="extended"
	fi
	# state = notified or renewed 
	
	if [ $state = "notified" -a \
	     $EPOCHNOW -gt $(($pid_notified + ($time_notified * $SECPERDAY) )) ]
	then
		# user has not responded in time
		state="kill"
	fi

	if [ "$user_kill" -gt "$user_renew" ]
	then
		if [ "$pid_randval" = "$user_key" ]
		then
			state="userkill"
		fi
	fi

	if [ $state = "unknown" ]
	then
		sendmail_debug "[PIM] pim_killer statemachine broken"
	fi

	if [ $state = "kill" -o $state = "userkill" ]
	then
		if [ $TESTMODE = false ]
		then
			# this is the only place a kill command is executed
			#sendmail_debug "[PIM] pim would have killed this one, check if correct"
			kill $pid_pid >/dev/null
		else
			echo "TESTMODE: $pid_elapsedtime -- cmd = $pid_cmdline"
			echo "TESTMODE: kill $pid_pid"
		fi
	
		if [ $state = "userkill" ]
		then
			sendmail_userkill >/dev/null
		else 
			sendmail_killed > /dev/null
		fi
		# wait a bit for the process to terminate
		sleep 10
		# update the current time
		EPOCHNOW=$(date +"%s")
		if [ ! -e /proc/$pid_pid ]
		then
			# process has been killed
			echo "pid_$state=\"$EPOCHNOW\"" >>$pidfile
			echo "pid_${state}_human=\"$(epoch2date $EPOCHNOW)\"" >>$pidfile
			quiet_mv $pim_datadir/${pid_machine}_$pid_pid $pim_olddatadir
			quiet_rm $pim_datadir/*_${pid_machine}_$pid_pid 
			quiet_mv $pim_useract/${pid_machine}_$pid_pid $pim_olddatadir/${pid_machine}_${pid_pid}_action
		fi	
	fi

done
